#!/bin/bash
cur_path=`pwd`/../

#Batch Size
batch_size=1
#网络名称，同目录名称
Network="CycleGAN_ID0521_for_PyTorch"
export RANKSIZE=8
RANK_ID_START=0
#Device数量，单卡默认为1
RankSize=8
#训练epoch，可选
train_epochs=200
#训练step
train_steps=
#学习率
learning_rate=

#参数配置
data_path=""

if [[ $1 == --help || $1 == --h ]];then
        echo "usage:./train_performance_1p.sh "
        exit 1
fi

for para in $*
do
        if [[ $para == --data_path* ]];then
                data_path=`echo ${para#*=}`
        fi
done

if [[ $data_path  == "" ]];then
        echo "[Error] para \"data_path\" must be config"
        exit 1
fi
start=$(date +%s)
##############执行训练##########
cd $cur_path
for((RANK_ID=$RANK_ID_START;RANK_ID<$((RANKSIZE+RANK_ID_START));RANK_ID++));
do
    #设置环境变量，不需要修改
    export ASCEND_DEVICE_ID=$RANK_ID
    echo "Device ID: $ASCEND_DEVICE_ID"
    export RANK_ID=$RANK_ID
    export LOCAL_RANK=$RANK_ID
    export MASTER_ADDR=127.0.0.1
    export MASTER_PORT=29688
    mkdir -p ./outputs/$data_path
    mkdir -p ./weights/$data_path/horse2zebra
    #创建DeviceID输出目录，不需要修改
    if [ -d ${cur_path}/test/output/${ASCEND_DEVICE_ID} ];then
        rm -rf ${cur_path}/test//output/${ASCEND_DEVICE_ID}
        mkdir -p ${cur_path}/test//output/$ASCEND_DEVICE_ID/ckpt
    else
        mkdir -p ${cur_path}/test/output/$ASCEND_DEVICE_ID/ckpt
    fi




#--decay_epochs 5 --apex
    nohup python3 train.py --dataset $data_path/horse2zebra --npu --epochs $train_epochs --batch-size $batch_size --apex --lr 0.0016 --device_id ${ASCEND_DEVICE_ID} > $cur_path/test/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log 2>&1 &
done
wait

end=$(date +%s)
e2etime=$(( $end - $start ))

sed -i "s|\r|\n|g" $cur_path/test/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log

#结果打印，不需要修改
echo "------------------ Final result ------------------"
#输出性能FPS，需要模型审视修
FPS=`grep fps $cur_path/test/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log|awk -F 'fps: ' '{print $2}'|awk -F ':' '{print$1}'|tail -n +3|awk '{sum+=$1} END {print sum/NR}'`

#打印，不需要修改
#echo "Final Performance images/sec : $FPS"
#echo "Final Training Duration sec : $e2etime"

#输出训练精度,需要模型审视修改
train_accuracy=None
#打印，不需要修改
#echo "Final Train Accuracy : ${train_accuracy}"

#性能看护结果汇总
#训练用例信息，不需要修改
BatchSize=${batch_size}
DeviceType=`uname -m`
CaseName=${Network}_bs${BatchSize}_${RankSize}'p'_'acc'

##获取性能数据，不需要修改
#吞吐量
ActualFPS=${FPS}
#单迭代训练时长
TrainingTime=`awk 'BEGIN{printf "%.2f\n",'1000'*'${batch_size}'/'${FPS}'}'`

#从train_$ASCEND_DEVICE_ID.log提取Loss到train_${CaseName}_loss.txt中，需要根据模型审视
grep Loss_D $cur_path/test/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log|awk -F 'Loss_D: ' '{print $2}'|awk '{print $1}'|awk '{if(length !=0) print $0}' > $cur_path/test/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt

#最后一个迭代loss值，不需要修改
ActualLoss=`awk 'END {print $1}' $cur_path/test/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt`

#关键信息打印到${CaseName}.log中，不需要修改
echo "Network = ${Network}" > $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
echo "RankSize = ${RankSize}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
echo "BatchSize = ${BatchSize}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
echo "DeviceType = ${DeviceType}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
echo "CaseName = ${CaseName}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
echo "ActualFPS = ${ActualFPS}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
echo "TrainingTime = ${TrainingTime}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
echo "ActualLoss = ${ActualLoss}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
echo "E2ETrainingTime = ${e2etime}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
echo "TrainAccuracy = ${ActualLoss}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log